/*
 * Created on 2004-11-12
 *
 *
 * Copyright: Bluejay Networks Inc. & Inquiry Management Systems Ltd.  2004
 *
 * This software is the proprietary information of Bluejay Networks Inc. & 
 * Inquiry Management Systems Ltd.  
 * Use is subject to license terms.
 */
package com.ims.iaa.html;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.io.Reader;
import java.net.*;
import java.util.List;
import java.util.LinkedList;
import java.util.HashMap;
import com.ims.iaa.JavaScript.*;
import com.ims.iaa.IaaUtil;
import com.ims.iaa.*;
/**
*
* This class pretreatment a html page, get rid of comment
* and get external script base on URL and then insert it to
* original position, such as:
*<script language="javascript" src="myurl"></script>
* this class will get script concetn form myurl, and then write
* this script like this:
* <script language="javascript">
* 	the content which get form myurl.
* </script>  
* 
* @author ShengMeng Bai
* 
* @version 1.0
* 
*/
public class HTMLPretreatment {
	//the original html page content
	private StringBuffer m_sbHtml;
	//new html page content
//	private StringBuffer m_newHtml;
	//the original html page content, all char had been converted to
	//lower case, so we can use it to judge script tag
//	private String m_sHtml;
	//the html page url, we can use it to compose script src url
	private URL m_baseURL;
	
	//use it to look for script tag
	private final static String SCRIPT_START 	= "<script";
	private final static String SCRIPT_END 		= "</script>";
	private final static int SCRIPT_END_LEN 	= SCRIPT_END.length();
	//use it to look for frame tag
	private final static String FRAME_START 	= "<frame";
	//private final static String FRAME_END 		= "</frame>";
	//private final static int FRAME_END_LEN 		= FRAME_END.length();
	//use it to look for frame tag
	private final static String IFRAME_START 	= "<iframe";
	private final static String IFRAME_END 		= "</iframe>";
	private final static int IFRAME_END_LEN 	= IFRAME_END.length();
	//use it to look for comment
	private final static String COMMENT_START 	= "<!--";
	private final static String COMMENT_END 	= "-->";
	private final static int COMMENT_END_LEN 	= COMMENT_END.length();
	//use it to look for noscript tag and noembed tag
	private final static String NOSCRIPT_START 	= "<noscript";
	private final static String NOSCRIPT_END 		= "</noscript>";
	private final static int NOSCRIPT_END_LEN 	= NOSCRIPT_END.length();
	private final static String NOEMBED_START 	= "<noembed";
	private final static String NOEMBED_END 		= "</noembed>";
	private final static int NOEMBED_END_LEN 	= NOEMBED_END.length();

	private final static String SRC	 			= " src=";
	private final static int SRC_LEN 			= SRC.length();
	
	private JavaScriptParser m_JSParser;
	private LinkedList m_SubPageList = new LinkedList();
	//connect to get the js file, we must set the userAgent
	private String m_sUserAgent = null;
	/**
	 * @param sHtml, the html page string
	 * @param baseURL, the html page URL 
	 */
	public HTMLPretreatment(String sHtml, URL baseURL, ImpAnalyzer impa,int specialJS)
	{
		this(new StringBuffer(sHtml), baseURL, impa, specialJS);
	}
	/**
	 * @param sbHtml, the html page StringBuffer
	 * @param baseURL, the html page URL 
	 */
	public HTMLPretreatment(StringBuffer sbHtml, URL baseURL, ImpAnalyzer impa,int specialJS)
	{
		m_baseURL = baseURL;
		m_sbHtml = sbHtml;
		if(IaaDebug.AuxFileDebug)
		{
			PrintStream out = null;
			try{
				String sFileName = PageAnalyzer.escapeURLString(baseURL);
	    		sFileName = IaaDebug.LogDirectory + "\\" + sFileName + ".js";
				out = new PrintStream( new FileOutputStream(sFileName));
			}catch (Exception e){
				//
			}
			/*m_JSParser = new JavaScriptParser(baseURL, impa, out);*/
			m_JSParser = JSParserFactory.getJSParser(baseURL,impa,out,specialJS);
		}else{
			/*m_JSParser = new JavaScriptParser(baseURL, impa);*/
			m_JSParser = JSParserFactory.getJSParser(baseURL,impa,null,specialJS);
		}
		//++add by ShengMengBai at 2005-04-04
	    m_sUserAgent = impa.getUserAgent();
		//--add by ShengMengBai at 2005-04-04
		pretreat();
		//ADDED BY yxy at 2008-08-12
		m_sbHtml = cleanForNoScript(m_sbHtml);
		//IaaUtil.log("the html after pretreat:");
		//IaaUtil.log(m_sbHtml);
	}
	/**
	 * Some pupup windwo need use parent's function,
	 * So we need pretreat the popup html in original opener's
	 * javascript envionment to parse html.
	 * 
	 * because this method will change the html content, so before call this method
	 * you must call #getDisposalHtml() to get old html content.
	 * 
	 * @param sbHtml, the popup html content.
	 * @param popURL, the popup URL
	 * @param iPopupIndex, the popup index in opener's popup index.
	 */
	public void pretreatPopup(StringBuffer sbHtml, URL popURL, int iPopupIndex){
		//change location object
		m_JSParser.changeLocation(popURL);
		//clear the document
		m_JSParser.clearDocument();
		//set the window.opener object;
		m_JSParser.ParseScriptContent("window.opener = this;");
		//clear old popup document content
		m_JSParser.clearPopupContent(iPopupIndex);
		//pre-process the html
		m_sbHtml = sbHtml;
		pretreat();
	//++some site show popup base on window.opener, in this instance, we should get the content form popup content 
		String sDoc = m_JSParser.getPopupContent(iPopupIndex);
		//if the popup use window.opener.write() to write the content,
		//get this content as popup content.
		if(!sDoc.equals("")){
			m_sbHtml = new StringBuffer(sDoc);
			//the document should be a sub-html, so need retreat it.
			pretreat();
			//clean the tag.
			m_sbHtml = cleanForPopup(m_sbHtml);
		}
	//--some site show popup base on window.opener.... 
	}
	
	//decrease the size and clean NOSCRIPT tag and NOEMBED tag 
	private static StringBuffer cleanForPopup(StringBuffer sbHtml)
	{
		//if the size is too large(more than 3K), if shouldn't be ad, don't process it
		int iLen = sbHtml.length();
		if(iLen < 3072){//if the size < 3K, will process it.
			StringBuffer newHtml;
			String sLow = sbHtml.toString().toLowerCase();
			//if the data count less than 2, return;
			if(getPopupAdDataCount(sLow) < 2){
				return sbHtml;
			}
			int iIndex = 0;
			//find noscript tag
			int iStart = sLow.indexOf(NOSCRIPT_START, iIndex);
			if(iStart > -1){
				newHtml = new StringBuffer(iLen);
				while(iStart > -1){
					int iEnd = sLow.indexOf(NOSCRIPT_END, iStart);
					if(iEnd > -1){
						//skip the noscript content
						newHtml.append(sbHtml.substring(iIndex, iStart));
						iIndex = NOSCRIPT_END_LEN + iEnd;
						//prepair next loop
						iStart = sLow.indexOf(NOSCRIPT_START, iIndex);
					}else{
						break;
					}
				}
				//add the remain content
				newHtml.append(sbHtml.substring(iIndex));
				
				sLow = newHtml.toString().toLowerCase();
				int iCount = getPopupAdDataCount(sLow);
				//if the data count less than 2, return;
				if(iCount < 1){
					return sbHtml;
				}else if(iCount == 1){
					return newHtml;
				}
				sbHtml = newHtml;
			}
			iIndex = 0;
			//find noembed tag
			iStart = sLow.indexOf(NOEMBED_START, iIndex);
			if(iStart > -1){
				newHtml = new StringBuffer(sbHtml.length());
				while(iStart > -1){
					int iEnd = sLow.indexOf(NOEMBED_END, iStart);
					if(iEnd > -1){
						//skip the noembed content
						newHtml.append(sbHtml.substring(iIndex, iStart));
						iIndex = NOEMBED_END_LEN + iEnd;
						//prepair next loop
						iStart = sLow.indexOf(NOEMBED_START, iIndex);
					}else{
						break;
					}
				}
				//add the remain content
				newHtml.append(sbHtml.substring(iIndex));
				
				sLow = newHtml.toString().toLowerCase();
				int iCount = getPopupAdDataCount(sLow);
				//if the data count less than 1, return old;
				if(iCount < 1){
					return sbHtml;
				}else if(iCount == 1){
					sbHtml = newHtml;
				}else{//if the count too more, return "";
		            IaaUtil.log("image or flash is too more, it shouldn't be AD.");
		            sbHtml = new StringBuffer("");
				}
			}
		}
		return sbHtml;
	}
	/*
	 * find the ad data(only img and flash) count
	 */
	private static int getPopupAdDataCount(String sHtml){
		int iCount = 0;
		int iStart = sHtml.indexOf("<img ");
		while(iStart != -1){
			iCount++;
			iStart = sHtml.indexOf("<img ", iStart+1);
		}
		iStart = sHtml.indexOf("<object ");
		while(iStart != -1){
			iCount++;
			iStart = sHtml.indexOf("<object ", iStart+1);
		}
		return iCount;
	}
	/**
	 * @return the pretreatmented html StringBuffer.
	 */
	public StringBuffer getDisposalHtml()
	{
		return m_sbHtml;
	}
	
	/**
	 * Some popup windwow open throught a URL 
	 * and some popup maybe open through blank URL
	 * and then through document.write to wrtite the content.
	 *  
	 * this method will get popup URL or popup content,
	 * this function get all the popup URL save it to a LinkedList
	 * and get all the popup content and save it to another LinkedList.
	 * 
	 * The two list will save to a hashmap, the URL list save as #MAP_POPUP_URL
	 * and the content list save as #MAP_POPUP_CONTENT
	 *
	 * @return HashMap boject
	 */
	public HashMap getPopupPage(){
		return m_JSParser.getPopup();
	}
	
	/**
	 * Get the sub page relative URL String List.
	 * The list conatin some String object, not URL object.
	 * this sub page show through <frame> or <iframe> tag in normal page.
	 * @return the list;
	 */
	public List getSubPage(){
		return m_SubPageList;
	}

	private void pretreat(){
		m_sbHtml = pretreatScript(m_sbHtml);
		m_sbHtml = pretreatSubPage(m_sbHtml);
	}
	//Parse javascript and get rid of comment
	//this function and pretreatComment function are combined,
	//the old function has bug, some script are delete as comment.
	private StringBuffer pretreatScript(StringBuffer sbHtml){
		int iStart = 0;
		int iCommentStart = 0;
		int iCommentEnd = 0;
		int iScriptStart = 0;
		int iScriptEnd = 0;
		//convert to lower case, so we can search script tag
		String sHtml = sbHtml.substring(0).toLowerCase();
		StringBuffer newHtml = new StringBuffer(sbHtml.length() + 1024);
		while(iCommentStart > -1 || iScriptStart > -1)
		{
			iCommentStart = getCommentStart(sHtml, iStart);
			if(iCommentStart > -1){
				iCommentEnd = getCommentEnd(sHtml, iCommentStart);
			}
			iScriptStart = getScriptStart(sHtml, iStart);
			if(iScriptStart > -1){
				iScriptEnd = getScriptEnd(sHtml, iScriptStart);
			}
			
			if(iCommentStart > -1 || ((iScriptStart > -1) && (iScriptEnd > iScriptStart))){
			//++if has comment or script
				if ((iScriptStart > -1) &&
					((iCommentStart > iScriptStart) || iCommentStart == -1) &&
					(iScriptEnd > iScriptStart)){
					//append content
					newHtml.append(sbHtml.substring(iStart, iScriptStart));
				//++if the comment is script comment
					//get script
					String sScript = sbHtml.substring(iScriptStart, iScriptEnd + SCRIPT_END_LEN);

					sScript = getScript(sScript);
					if(sScript.length() > 0){
						newHtml.append(sScript);
					}
					iStart = iScriptEnd + SCRIPT_END_LEN;
				//--if the comment is script comment
				}else{
				//++is real comment, delete it
					//append content
					newHtml.append(sbHtml.substring(iStart, iCommentStart));
					//seek the pointer and skip comment
					iStart = iCommentEnd + COMMENT_END_LEN;
				//--is real comment, delete it
				}
			//--if has comment or script
			}else{
				newHtml.append(sbHtml.substring(iStart));
				break;
			}
		}
		return newHtml;
	}
	
	/*
	 * Get the sub page from <frame> and <iframe> tag
	 */
	private StringBuffer pretreatSubPage(StringBuffer sbHtml){
		int iStart = 0;
		int iFrameStart = 0;
		int iIframeStart = 0;
		int iIframeEnd = 0;
		//convert to lower case, so we can search frame tag and iframe tag
		String sHtml = sbHtml.substring(0).toLowerCase();
		StringBuffer newHtml = new StringBuffer(sbHtml.length());
		TagAttributeParser tagParser = new TagAttributeParser();
		while(iFrameStart > -1)
		{
			iFrameStart = getFrameStart(sHtml, iStart);
			if(iFrameStart > -1){
				//get the end of the tag
				int iIndex = sHtml.indexOf('>', iFrameStart);
				iStart=iIndex; // Feb. 19, 2005 - add this to fix the infinite loop bug - need to advance iStart to iIndex
				if(iIndex > -1){
					//get the tag string
					String sTag = sbHtml.substring(iFrameStart, iIndex + 1);
					try{
						//get the src
						tagParser.Parse(sTag);
						String sSrc = tagParser.getValue("src");
						if(sSrc != null){
							m_SubPageList.add(sSrc);
						} /* else{
							IaaUtil.log("Cannot get src form : " + sTag);
						} */ // Feb. 19, 2005 - comment it out since it is not needed, frame src can be defined after frameset 
					}catch (Exception e){
						IaaUtil.log(sTag + " cannot be parsed, message is: " + e.toString());
					}
				}
			}
		}
		iStart = 0;
		while(iIframeStart > -1)
		{
			iIframeStart = getIframeStart(sHtml, iStart);
			if(iIframeStart > -1){
				//get the end of the tag
				int iIndex = sHtml.indexOf('>', iIframeStart);
				if(iIndex > -1){
					//get the tag string
					String sTag = sbHtml.substring(iIframeStart, iIndex + 1);
					try{
						//get the src
						tagParser.Parse(sTag);
						String sSrc = tagParser.getValue("src");
						if(sSrc != null){
							m_SubPageList.add(sSrc);
						}else{
							IaaUtil.log("Cannot get src form : " + sTag);
						}
					}catch (Exception e){
						IaaUtil.log(sTag + " cannot be parse, message is: " + e.toString());
					}
				}
				iIframeEnd = getIframeEnd(sHtml, iIframeStart);
				newHtml.append(sbHtml.substring(iStart, iIframeStart));
				//skip the <iframe> tag
				if(iIframeEnd != -1){
					iStart = iIframeEnd + IFRAME_END_LEN;
				}else{
					if(iIndex > -1){
						iStart = iIndex + 1;
					}else{
						break;
					}
				}
			}else{
				newHtml.append(sbHtml.substring(iStart));
			}
		}
		return newHtml;
	}
	/*
	 * convert <script> tag to real html
	 * "document.write(...)" will convert to real html
	 * "window.open(...)" will convert to popup page.
	 */
	private String getScript(String sScript)
	{
		if(m_JSParser.ParseScriptTag(sScript)){
			//get the html form document.write();
			String sDoc = m_JSParser.getDocument();
			//clear the parser cache
			m_JSParser.clearDocument();
			String sLowerCase = sDoc.toLowerCase();
			//if has sub javascript, parse it again.
			if(this.getScriptStart(sLowerCase, 0) > -1){
				sDoc = rePretreatScript(sDoc, sLowerCase);
			}
			return sDoc;
		}else{
			return "";
		}
	}
	/*
	 * Some javascript write in javascript, so need retreat it again
	 * 
	 * Parse javascript
	 */
	private String rePretreatScript(String sHtml, String sLowerCaseHtml){
		int iStart = 0;
		int iScriptStart = 0;
		int iScriptEnd = 0;
		StringBuffer newHtml = new StringBuffer(sHtml.length() + 1024);
		
		iScriptStart = getScriptStart(sLowerCaseHtml, iStart);
		iScriptEnd = getScriptEnd(sLowerCaseHtml, iStart);
		
	//++get all script from iStart and comment
		while(iScriptStart > -1 && iScriptEnd > -1){
			//append content
			newHtml.append(sHtml.substring(iStart, iScriptStart));
			//append script
			String sScript = sHtml.substring(iScriptStart, iScriptEnd + SCRIPT_END_LEN);
			newHtml.append(getScript(sScript));
			
			iStart = iScriptEnd + SCRIPT_END_LEN;
			iScriptStart = getScriptStart(sLowerCaseHtml, iStart);
			iScriptEnd = getScriptEnd(sLowerCaseHtml, iStart);
		}
	//--get all script from iStart and comment
		newHtml.append(sHtml.substring(iStart));
		return newHtml.substring(0);
	}
	/*
	 * get the firt comment start sign (<!--) occured position form specified position
	 */
    private int getCommentStart(String sbContent, int iStart)
	{
		return sbContent.indexOf(COMMENT_START, iStart);
	}
	/*
	 * get the firt comment end sign (-->) occured position form specified position
	 */
	private int getCommentEnd(String sbContent, int iStart)
	{
		return sbContent.indexOf(COMMENT_END, iStart);
	}
	/*
	 * get the firt script start sign (<script) occured position form specified position
	 */
	private int getScriptStart(String sbContent, int iStart)
	{
		return sbContent.indexOf(SCRIPT_START, iStart);
	}
	/*
	 * get the firt script end sign (</script>) occured position form specified position
	 */
	private int getScriptEnd(String sbContent, int iStart)
	{
		return sbContent.indexOf(SCRIPT_END, iStart);
	}
	/*
	 * get the firt frame start sign (<frame) occured position form specified position
	 */
    private int getFrameStart(String sbContent, int iStart)
	{
		return sbContent.indexOf(FRAME_START, iStart);
	}
	/*
	 * get the firt iframe start sign (<iframe) occured position form specified position
	 */
    private int getIframeStart(String sbContent, int iStart)
	{
		return sbContent.indexOf(IFRAME_START, iStart);
	}
	/*
	 * get the firt iframe end sign (</iframe>) occured position form specified position
	 */
    private int getIframeEnd(String sbContent, int iStart)
	{
		return sbContent.indexOf(IFRAME_END, iStart);
	}
	public static void main( String[] args ) {
		if(args.length == 0){
			System.out.println("You must input a site.");
			System.exit(3);
		}
		HTMLPretreatment html;
        try {
        	Class.forName("com.ims.iaa.IaaDebug");
    		URL url = URLUtil.getURL(args[0]);
    		URLConnection uc = url.openConnection();
			//++add by ShengMengBai at 2005-04-04
			uc.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)");
			//--add by ShengMengBai at 2005-04-04
    		StringBuffer sbHtml = new StringBuffer();
            IaaUtil.log("Start pre-read html: " + uc.getURL());
            // read html content to StringBuffer.
            Reader preReader = new InputStreamReader(uc.getInputStream());
            int bufSize = 51200;//50k
            char[] buf = new char[bufSize];
			while(true){
                int moved = preReader.read(buf);
                if ( moved < 0 ) break;
                sbHtml.append(buf, 0, moved);
			}
			preReader.close();
			IaaUtil.log("End pre-read html: " + uc.getURL());
			
			html = new HTMLPretreatment(sbHtml, url, new ImpAnalyzerImpl(),0);
        } catch (IOException ioe) {
        	IaaUtil.printStackTrace(ioe);
            //IaaUtil.log("Pre-read html error: " + ioe);
            return;
        } catch (Exception e) {
        	IaaUtil.printStackTrace(e);
            //IaaUtil.log("Pre-read html error: " + e);
            return;
        }
		System.out.println("Ok");
	}
//	decrease the size and clean NOSCRIPT tag and NOEMBED tag 
	private static StringBuffer cleanForNoScript(StringBuffer sbHtml)
	{
		//if the size is too large(more than 3K), if shouldn't be ad, don't process it
		int iLen = sbHtml.length();

			StringBuffer newHtml;
			String sLow = sbHtml.toString().toLowerCase();
		
			int iIndex = 0;
			//find noscript tag
			int iStart = sLow.indexOf(NOSCRIPT_START, iIndex);
			if(iStart > -1){
				newHtml = new StringBuffer(iLen);
				while(iStart > -1){
					int iEnd = sLow.indexOf(NOSCRIPT_END, iStart);
					if(iEnd > -1){
						//skip the noscript content
						newHtml.append(sbHtml.substring(iIndex, iStart));
						iIndex = NOSCRIPT_END_LEN + iEnd;
						//prepair next loop
						iStart = sLow.indexOf(NOSCRIPT_START, iIndex);
					}else{
						break;
					}
				}
				//add the remain content
				newHtml.append(sbHtml.substring(iIndex));
			    sbHtml = newHtml;
			}

		return sbHtml;
	}
}
